# -*- coding: utf-8 -*-

import re
import sys
from urlparse import urljoin

from bs4 import BeautifulSoup
from scrapy import Request
from scrapy.spiders import Spider

from realEstateCommittee.commons import _REPLACE_NBSP
from realEstateCommittee.items import RealestatecommitteeItem

reload(sys)
sys.setdefaultencoding('utf8')


class MyluanItem_Spider(Spider):
    name = "RealEstateCommittee_YJ_LuAn2"

    color_to_status = {"#F0FFB6": u"预售",
                       "#FFFF66": u"预定",
                       "#82B4FF": u"签约",
                       "#96E686": u"备案",
                       "#E5E5E5": u"自留",
                       "#FFCCFF": u"回迁",
                       "#CC3399": u"抵押",
                       "Salmon": u"政府回购",
                       "#FF0000": u"查封",
                       "#6666CC": u"限制",
                       "#7030A0": u"处理",
                       "#FF8D00": u"不可售",
                       "#CFE1E9": u"历史遗留",
                       "#646464": u"其他",
                       "#FFFFFF": u"地下车库",
                       "#03E4BC": u"公厕"}

    SHOW_PROJECT_INFO_URL = "show_project_info_url"  # 查看项目概况
    PROJECT_SUM_INFO_URL = "project_sum_info_url"  # 查看项目统计
    SHOW_PRESELL_CERT_INFO_URLS = "show_presell_cert_info_urls"  # 许可证号
    SHOW_BUILD_SALE_INFO_URL = "show_build_sale_info_url"  # 预售栋

    # 要提取的27个字段如下:
    # ------------ parse函数部分: --------------------------------------------
    PROJECT = "project"  # ："项目名称"
    # --------------------------------------------------------------------------------

    # ------------ parse_project_info函数部分: (查看项目概况)--------------------------------------------
    PROJECT_ADDR = "project_addr"  # : "项目地址",
    PROPERTY_COMPANY = "property_company"  # :"物业公司",
    PLOT_RATIO = "plot_ratio"  # :"容积率",
    GREEN_RATIO = "green_ratio"  # :"绿化率",
    DECORATE_TYPE = "decorate_type"  # :"装修类型",
    # --------------------------------------------------------------------------------

    # --------- parse_project_sum_info_url函数部分: (查看项目统计) -------------------
    BUILDING_NO = "building_no"  # :"楼号/幢号",
    RESIDENCE_SET = "residence_set"  # :"本期住宅总套数",
    RESIDENCE_AREA = "residence_area"  # :"本期住宅总面积",
    SOLD_RESIDENCE_SET = "sold_residence_set"  # :"本期已售住宅总套数",
    SOLD_RESIDENCE_AREA = "sold_residence_area"  # :"本期已售住宅总面积",
    UNSOLD_RESIDENCE_SET = "unsold_residence_set"  # :"本期未售住宅总套数",
    UNSOLD_RESIDENCE_AREA = "unsold_residence_area"  # :"本期未售住宅总面积",
    # --------------------------------------------------------------------------------

    # ---------- parse_show_project_presell_certinfo_url函数部分: ---------------------------------
    PRESALE_PERMIT = "presale_permit"  # :"预售证号码",
    DEVELOPER = "developer"  # :"开发商名称",
    OPEN_TIME = "open_time"  # : "开盘时间",
    BUILD_NO = "build_no"  # :"本期住宅楼号/幢号",
    BUILDING_RESIDENCE_NUM = "building_residence_num"  # :"本幢楼住宅套数",
    PROPERTY_TYPE = "property_type"  # :"物业类型/房屋类型",
    # --------------------------------------------------------------------------------

    # -------- parse_show_build_sale_info_url函数部分: ---------------------------------
    UNIT_NO = "unit_no"  # :"单元号",
    FLOOR = "floor"  # :"楼层",
    HOUSE_NO = "house_no"  # :"房号",
    HOUSE_TYPE = "house_type"  # :"房型",
    HOUSE_AREA = "house_area"  # :"房屋建筑面积",
    INNER_AREA = "inner_area"  # :"套内面积",
    PROPERTY_NATURE = "property_nature"  # :"物业性质",
    SALE_STATUS = "sale_status"  # :"销售状态"
    # --------------------------------------------------------------------------------
    start_urls = ["http://fcj.luan.gov.cn/ShowBuild.aspx"]

    def parse(self, response):  # 解析ShowBuild页面,再跳转到ShowSection页面

        # PROJECT = "project"  "项目名称"

        meta = response.meta
        soup = BeautifulSoup(response.text, "html5lib")
        projects_tag = soup.find_all("a", attrs={"href": re.compile("ShowSection")})
        self.logger.info(response.url)
        for project_tag in projects_tag:
            project_name = project_tag.get_text()
            meta[self.PROJECT] = project_name
            project_url = urljoin(response.url, project_tag.get("href"))
            self.logger.info("项目名称:%s → %s" % (project_name, project_url))
            yield Request(project_url, callback=self.parse_show_section_url, meta=meta, dont_filter=True)

    def parse_show_section_url(self, response):

        """"
            获取项目概况链接 show_project_info_url,
            项目统计 project_sum_info_url,
            许可证号 show_presell_cert_info_url,进入许可证链接后,可以获取预售栋链接show_build_sale_info_url

        """
        meta = response.meta
        soup = BeautifulSoup(response.text, "html5lib")
        cert_urls = list()
        if soup.find("a", attrs={"href": re.compile("ProjectID")}):
            meta[self.SHOW_PROJECT_INFO_URL] = urljoin(response.url,
                                                       soup.find("a", attrs={"href": re.compile(u"ProjectID")}).get(
                                                           "href"))  # 项目概况链接
        if soup.find("td", id="ctl00_ContentPlaceHolder1_tdProjectSum").a:
            append_url = soup.find("td", id="ctl00_ContentPlaceHolder1_tdProjectSum").a.get("href")
            meta[self.PROJECT_SUM_INFO_URL] = urljoin(response.url, append_url)  # 项目统计链接

        cert_table_tag = soup.find("table", attrs={"border": 0, "style": True, "cellpadding": 0})
        if cert_table_tag.find_all("a", attrs={"href": re.compile("DocID")}):
            for row in cert_table_tag.find_all("a", attrs={"href": re.compile("DocID")}):
                cert_urls.append(urljoin(response.url, row.get("href")))  # 许可证号链接
                meta[self.SHOW_PRESELL_CERT_INFO_URLS] = cert_urls
        if meta.get("show_project_info_url"):
            yield Request(meta[self.SHOW_PROJECT_INFO_URL], callback=self.parse_project_info, meta=meta,
                          dont_filter=True)

    def parse_project_info(self, response):  # 项目概况

        """
            PROJECT_ADDR = "project_addr"  # : "项目地址",
            PROPERTY_COMPANY = "property_company"  # :"物业公司",
            PLOT_RATIO = "plot_ratio"  # :"容积率",
            GREEN_RATIO = "green_ratio"  # :"绿化率",
            DECORATE_TYPE = "decorate_type"  # :"装修类型",
        """
        meta = response.meta
        soup = BeautifulSoup(response.text, "html5lib")
        meta[self.PROJECT_ADDR] = _REPLACE_NBSP(soup.find("span", id="ProjectAddress").get_text())
        meta[self.PROPERTY_COMPANY] = _REPLACE_NBSP(soup.find("span", id="EstateManageCompany").get_text())
        meta[self.PLOT_RATIO] = _REPLACE_NBSP(soup.find("span", id="CubageRate").get_text())
        meta[self.GREEN_RATIO] = _REPLACE_NBSP(soup.find("span", id="GreenRate").get_text())
        meta[self.DECORATE_TYPE] = _REPLACE_NBSP(soup.find("span", id="DeliveStandard").get_text())

        yield Request(meta[self.PROJECT_SUM_INFO_URL], callback=self.parse_project_sum_info_url, meta=meta,
                      dont_filter=True)

    def parse_project_sum_info_url(self, response):  # 项目统计

        """"
            BUILDING_NO = "building_no"  # :"楼号/幢号",
            RESIDENCE_SET = "residence_set"  # :"本期住宅总套数",
            RESIDENCE_AREA = "residence_area"  # :"本期住宅总面积",
            SOLD_RESIDENCE_SET = "sold_residence_set"  # :"本期已售住宅总套数",
            SOLD_RESIDENCE_AREA = "sold_residence_area"  # :"本期已售住宅总面积",
            UNSOLD_RESIDENCE_SET = "unsold_residence_set"  # :"本期未售住宅总套数",
            UNSOLD_RESIDENCE_AREA = "unsold_residence_area"  # :"本期未售住宅总面积",
        """
        meta = response.meta
        soup = BeautifulSoup(response.text, "html5lib")
        for row in soup.find_all("tr", attrs={"align": None, "style": True}):
            columns = [_REPLACE_NBSP(item.get_text()) for item in row.find_all("td")]
            meta[self.BUILDING_NO] = columns[0].strip()
            meta[self.RESIDENCE_SET] = columns[1]
            meta[self.RESIDENCE_AREA] = columns[2]
            meta[self.UNSOLD_RESIDENCE_SET] = columns[3]
            meta[self.UNSOLD_RESIDENCE_AREA] = columns[4]
            meta[self.SOLD_RESIDENCE_SET] = float(columns[11]) + float(columns[13]) + float(columns[15]) + float(
                columns[17])
            meta[self.SOLD_RESIDENCE_AREA] = \
                float(columns[12]) + float(columns[14]) + float(columns[16]) + float(columns[18])
            for i in meta[self.SHOW_PRESELL_CERT_INFO_URLS]:
                if i:
                    yield Request(i, callback=self.parse_show_project_presell_certinfo_url, meta=meta,
                                  dont_filter=True)

    def parse_show_project_presell_certinfo_url(self, response):

        """"
            PRESALE_PERMIT = "presale_permit"  # :"预售证号码",
            DEVELOPER = "developer"  # :"开发商名称",
            OPEN_TIME = "open_time"  # : "开盘时间",
            BUILD_NO = "build_no"  # :"本期住宅楼号/幢号",
            BUILDING_RESIDENCE_NUM = "building_residence_num"  # :"本幢楼住宅套数",
            PROPERTY_TYPE = "property_type"  # :"物业类型/房屋类型",
        """
        meta = response.meta
        soup = BeautifulSoup(response.text, "html5lib")
        meta[self.PRESALE_PERMIT] = _REPLACE_NBSP(soup.find("span", id="证书编号").get_text())
        meta[self.DEVELOPER] = _REPLACE_NBSP(soup.find("span", id="预售单位").get_text())
        meta[self.OPEN_TIME] = _REPLACE_NBSP(soup.find("span", id="开盘日期").get_text())
        meta[self.BUILD_NO] = _REPLACE_NBSP(soup.find("span", id="txtDong").get_text())
        meta[self.BUILDING_RESIDENCE_NUM] = _REPLACE_NBSP(soup.find("span", id="预售套数").get_text())
        meta[self.PROPERTY_TYPE] = _REPLACE_NBSP(soup.find("span", id="性质").get_text())

        meta[self.SHOW_BUILD_SALE_INFO_URL] = urljoin(response.url, soup.find("span", id="txtDong").a.get("href"))
        yield Request(meta[self.SHOW_BUILD_SALE_INFO_URL], callback=self.parse_show_build_sale_info_url, meta=meta,
                      dont_filter=True)

    def parse_show_build_sale_info_url(self, response):

        """""
            UNIT_NO = "unit_no"  # :"单元号",
            FLOOR = "floor"  # :"楼层",
            HOUSE_NO = "house_no"  # :"房号",
            HOUSE_TYPE = "house_type"  # :"房型",
            HOUSE_AREA = "house_area"  # :"房屋建筑面积",
            INNER_AREA = "inner_area"  # :"套内面积",
            PROPERTY_NATURE = "property_nature"  # :"物业性质",
            SALE_STATUS = "sale_status"  # :"销售状态"
        """
        meta = response.meta
        soup = BeautifulSoup(response.text, "html5lib")
        tag = soup.find("table", id="Room1__room_map_")
        if tag:
            for row in soup.find("table", id="Room1__room_map_").find_all("tr")[1:]:
                meta[self.FLOOR] = row.td.get_text().strip()
                unit_list = [unit.get_text() for unit in
                             soup.find("table", id="Room1__room_map_").tr.find_all("td")[1:]]
                for index, house in enumerate(row.find_all("td", attrs={"title": True}, recursive=False)):
                    title = house.get("title")
                    house_infos = [re.findall(u"：(.*)", item)[0] for item in title.split("\n")]
                    sale_status = self.color_to_status[house.get("bgcolor")]  # 销售状态
                    meta[self.UNIT_NO] = unit_list[index]
                    meta[self.HOUSE_NO] = house_infos[0]
                    meta[self.HOUSE_TYPE] = house_infos[1]
                    meta[self.INNER_AREA] = house_infos[2]
                    meta[self.HOUSE_AREA] = house_infos[3]
                    meta[self.PROPERTY_NATURE] = house_infos[4]
                    meta[self.SALE_STATUS] = sale_status

                    self.logger.info(
                        "项目名称:%s\n,项目地址:%s\n,物业公司:%s\n,容积率:%s\n,绿化率:%s\n,装修类型:%s\n,"
                        "楼号/幢号:%s\n,本期住宅总套数:%s\n,本期住宅总面积:%s\n,本期已售住宅总套数:%s\n,"
                        "本期已售住宅总面积:%s\n,本期未售住宅总套数:%s\n,本期未售住宅总面积:%s\n,预售证号码:%s\n,"
                        "开发商名称:%s\n,开盘时间:%s\n,本期住宅楼号/幢号:%s\n,本幢楼住宅套数:%s\n,"
                        "物业类型/房屋类型:%s\n,单元号:%s\n,楼层:%s\n,房号:%s\n,房型:%s\n,房屋建筑面积:%s\n,"
                        "套内面积:%s\n,物业性质:%s\n,销售状态:%s\n"
                        % (
                            meta[self.PROJECT], meta[self.PROJECT_ADDR], meta[self.PROPERTY_COMPANY],
                            meta[self.PLOT_RATIO],
                            meta[self.GREEN_RATIO], meta[self.DECORATE_TYPE], meta[self.BUILDING_NO],
                            meta[self.RESIDENCE_SET], meta[self.RESIDENCE_AREA], meta[self.SOLD_RESIDENCE_SET],
                            meta[self.SOLD_RESIDENCE_AREA], meta[self.UNSOLD_RESIDENCE_SET],
                            meta[self.UNSOLD_RESIDENCE_AREA], meta[self.PRESALE_PERMIT], meta[self.DEVELOPER],
                            meta[self.OPEN_TIME], meta[self.BUILD_NO], meta[self.BUILDING_RESIDENCE_NUM],
                            meta[self.PROPERTY_TYPE], meta[self.UNIT_NO], meta[self.FLOOR], meta[self.HOUSE_NO],
                            meta[self.HOUSE_TYPE], meta[self.HOUSE_AREA], meta[self.INNER_AREA],
                            meta[self.PROPERTY_NATURE],
                            meta[self.SALE_STATUS]))
                    self.logger.info("-----------------------------------------")
                    yield self.meta_to_item(meta)
        pass

    def meta_to_item(self, meta):
        item = RealestatecommitteeItem()
        for field in RealestatecommitteeItem.fields:
            if field not in meta.keys():
                item[field] = ""
            else:
                item[field] = meta[field]
        return item
